{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "56227069",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6e5819f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(nn.Linear(20, 256), nn.ReLU(), nn.Linear(256, 10))\n",
    "X = torch.rand(2, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "db839d6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.1539,  0.2439,  0.1419,  0.1061,  0.2851, -0.0341, -0.0266,  0.1835,\n",
       "          0.0846, -0.1791],\n",
       "        [-0.0586,  0.1759,  0.1577,  0.0864,  0.2482,  0.0832, -0.0386,  0.1961,\n",
       "          0.0206, -0.2415]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a0505dd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "F.relu()是函数调用，一般使用在foreward函数。\n",
    "nn.ReLU()是模块调用，一般在定义网络层的时候使用。\n",
    "当用print(net)输出时，会有nn.ReLU()层，而F.ReLU()是没有输出的。\n",
    "'''\n",
    "class MLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.hidden = nn.Linear(20, 256)\n",
    "        self.out = nn.Linear(256, 10)\n",
    "        \n",
    "    def forward(self, X):\n",
    "        return self.out(F.relu(self.hidden(X)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3595c517",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.1219,  0.2362, -0.0249, -0.2978, -0.0079, -0.0140,  0.0063,  0.0402,\n",
       "          0.1832,  0.0780],\n",
       "        [-0.1683,  0.2141, -0.0024, -0.2245,  0.0317, -0.0455,  0.0575, -0.0456,\n",
       "          0.2931,  0.1748]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = MLP()\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fd25562a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MySequential(nn.Module):\n",
    "    def __init__(self, *args):\n",
    "        super().__init__()\n",
    "        for idx, block in enumerate(args):\n",
    "            self._modules[str(idx)] = block\n",
    "            \n",
    "    def forward(self, X):\n",
    "        for block in self._modules.values():\n",
    "            X = block(X)\n",
    "        return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0f5f7757",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.2353, -0.0136,  0.2088, -0.0389,  0.0985, -0.0635,  0.0584, -0.1806,\n",
       "         -0.0005, -0.2087],\n",
       "        [-0.1169,  0.0521,  0.1133, -0.0147,  0.0965, -0.0599,  0.1453, -0.1749,\n",
       "          0.0172, -0.1124]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = MySequential(nn.Linear(20, 256), nn.ReLU(), nn.Linear(256, 10))\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a113dff4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FixedHiddenMLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.rand_weight = torch.rand((20, 20), requires_grad=False) # 0-1均匀分布\n",
    "        self.linear = nn.Linear(20, 20)\n",
    "        \n",
    "    def forward(self, X):\n",
    "        X = self.linear(X)\n",
    "        X = F.relu(torch.mm(X, self.rand_weight) + 1)\n",
    "        X = self.linear(X)\n",
    "        while X.abs().sum() > 1:\n",
    "            X /= 2\n",
    "        return X.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "304b589e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-0.0030, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = FixedHiddenMLP()\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "15945017",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NestMLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(nn.Linear(20, 64), nn.ReLU(),\n",
    "                                 nn.Linear(64, 32), nn.ReLU())\n",
    "        self.linear = nn.Linear(32, 16)\n",
    "        \n",
    "    def forward(self, X):\n",
    "        return self.linear(self.net(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "27780b42",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.1826, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = nn.Sequential(NestMLP(), nn.Linear(16, 20), FixedHiddenMLP())\n",
    "net(X)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python(d2l)",
   "language": "python",
   "name": "d2l"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
